# -*- coding: UTF-8 -*-
# cython: language_level=3
import multiprocessing
from concurrent import futures

from tqdm import tqdm

from base.base_parser import add_icp, fenlei
from db.mysqldb import MysqlDB
from utils.confident import site_checker
from utils.get_domain_info import get_website_info
from utils.ip_area import find_ip


def add_root_url(host, home_page, sid):
    db = MysqlDB()

    website_info = get_website_info(home_page, host=host)
    ip = website_info['ip']

    try:
        assert ip
        country, province_name, city_name, isp = find_ip(ip)
        if country and country != '中国':
            province_name = country + province_name
    except:
        province_name = ''
        city_name = ''

    if host.endswith('.gov.cn') or host.endswith('.edu.cn') or host.endswith('.gov') or host.endswith('.edu'):
        status = 8
    else:
        if website_info['visit_status'] == 0:
            status = fenlei(website_info['title'], icp=website_info['icp'], province_name=province_name)
        else:
            status = 3

    add_icp(status=status, province_name=province_name, city_name=city_name, **website_info)
    db.update(f"update todo_urls set status=2 where id={sid}")


def add_root_url_t(INPUT_URLS, n=None):
    len_urls = len(INPUT_URLS)
    # print('添加url数量： ', len_urls)
    p_num = min(len_urls, 40)
    if not p_num:
        return

    tasks, results = [], []
    with futures.ThreadPoolExecutor(max_workers=p_num) as executor:
        for sid, home_page, host in tqdm(INPUT_URLS, total=len_urls, desc='add_task'):
            tasks.append(executor.submit(add_root_url, host, home_page, sid))

        for task in tqdm(futures.as_completed(tasks), total=len_urls, desc=f'update_mongo-{n}'):
            # results.append(task.result())
            pass
    # print(results)


def add_root_url_main(max_workers=10):
    db = MysqlDB()
    db.update("update todo_urls set status=1 where status=0")
    INPUT_URLS = db.find("select id,home_page,host from todo_urls where status=1")
    len_urls = len(INPUT_URLS)
    print('添加url数量： ', len_urls, 'max_workers:', max_workers)

    if not len_urls:
        return
    step = int(len_urls / max_workers) + 1
    INPUT_URLS = [INPUT_URLS[i:i + step] for i in range(0, len_urls, step)]

    tasks, results = [], []
    with futures.ProcessPoolExecutor(max_workers=max_workers) as executor:
        for i, data in enumerate(INPUT_URLS):
            tasks.append(executor.submit(add_root_url_t, data, i))
        for task in tqdm(futures.as_completed(tasks), total=max_workers, desc='总进度'):
            # results.append(task.result())
            pass
    # print(results)


def check(site_id, url, host):
    db = MysqlDB()
    db.update(f"update tab_video_site_info set NO_LICENSE_SITE_STATUS=1 where ID={site_id}")
    site_info = site_checker(url, host)
    try:
        site_confidence = site_info['site_Confidence']
        page_num = site_info['page_num']
    except:
        pass
    else:
        if site_confidence > 2000 and page_num >= 3:
            CHECK_TYPE = 0
        else:
            CHECK_TYPE = 2

        db.update(f"update tab_video_site_info "
                  f"set NO_LICENSE_SITE_STATUS=2,CHECK_TYPE={CHECK_TYPE},NO_LICENSE_SITE_CONFIDENCE={site_confidence} "
                  f"where ID={site_id}")


def check_main(man_input=None):
    """
    检测主函数
    """
    db = MysqlDB()
    db.update(f"update tab_video_site_info set NO_LICENSE_SITE_STATUS=10 where SITE_TYPE=3 and CHECK_TYPE=10 and NO_LICENSE_SITE_STATUS=0 limit 100")

    check_urls = db.find(
        "select ID,URL,DOMAIN from tab_video_site_info where SITE_TYPE=3 and CHECK_TYPE=10 and NO_LICENSE_SITE_STATUS=10")
    if not check_urls:
        return

    print('待检查url数量： ', len(check_urls))
    p_num = len(check_urls) if len(check_urls) < 4 else 4
    p = multiprocessing.Pool(p_num)
    print("-----start-----")
    for site_id, url, host in check_urls:
        p.apply_async(check, (site_id, url, host,))
    p.close()
    p.join()
    print("-----end-----")


if __name__ == "__main__":
    add_root_url_main(max_workers=5)
    # check_main()
